"""
import socket
import urllib.request
import urllib.parse
response = urllib.request.urlopen('https://www.python.org')
print(response.status)  # 响应的状态码
print(response.getheaders())    # 响应头的信息
print(response.getheader('Server')) # 响应头中Server的值
"""
# data
import http.cookiejar
import urllib.request

"""
import socket
import urllib.request
import urllib.parse
data = bytes(urllib.parse.urlencode({'hello':'world'}),encoding='utf-8')
response = urllib.request.urlopen('http://httpbin.org/post', data=data)
print(response.read())
"""
# timeout
"""
import socket
import urllib.request
import urllib.parse
try:
    response = urllib.request.urlopen('http://httpbin.org/get', timeout=1)
    print(response.read())
except urllib.error.URLError as e:
    if isinstance(e.reason,socket.timeout):
        print('TIME OUT')
"""
# Request
"""
from urllib import request,parse
url = 'http://httpbin.org/post'
headers = {
    'User-Agent': 'Mozilla/4.0 (compatible; MSIE 5.5; windows NT)',
    'Host': 'httpbin.org',
}
dict = {
    'name': 'Germy'
}

# data = bytes(parse.urlencode(dict),encoding='utf-8')
data = bytes(parse.urlencode(dict),encoding='utf-8')
req = request.Request(url=url,data=data,headers=headers,method='POST')
response = request.urlopen(req)
print(response.read().decode('utf-8'))
"""
# 高级用法
## 验证
"""
from urllib.request import HTTPPasswordMgrWithDefaultRealm,HTTPBasicAuthHandler,build_opener
from urllib.error import URLError

username = 'username'
password = 'password'
url = 'http://localhost:5000/'

p = HTTPPasswordMgrWithDefaultRealm()
p.add_password(None,url,username,password)
auth_handler = HTTPBasicAuthHandler(p)
opener = build_opener(auth_handler)

try:
    result = opener.open(url)
    html = result.read().decode('utf-8')
    print(html)
except URLError as e:
    print(e.reason)
"""
## 代理
"""
from urllib.error import URLError
from urllib.request import ProxyHandler,build_opener

proxy_handle = ProxyHandler({
    'http':'http//127.0.0.1:9743',
    'https': 'https://127.0.0.1:9743'
})

opener = build_opener(proxy_handle)
try:
    response = opener.open('https://www.baidu.com')
    print(response.read().decode('utf-8'))
except URLError as e:
    print(e.reason)
"""
## Cookies
"""
import http.cookiejar, urllib.request

cookies = http.cookiejar.CookieJar()
handler = urllib.request.HTTPCookieProcessor(cookies)
opener = urllib.request.build_opener(handler)
response = opener.open('http://www.baidu.com')

for item in cookies:
    print(item.name + '   =   ' + item.value)
"""
"""
filename = 'cookies.txt'
cookie = http.cookiejar.MozillaCookieJar(filename)
handle = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handle)
response = opener.open('http://www.baidu.com')
cookie.save(ignore_discard=True,ignore_expires=True)
"""
"""
filename = 'cookiesLWP.txt'
cookie = http.cookiejar.LWPCookieJar(filename)
handle = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handle)
response = opener.open('http://www.baidu.com')
cookie.save(ignore_discard=True, ignore_expires=True)
"""
"""
cookie = http.cookiejar.LWPCookieJar()
cookie.load('cookiesLWP.txt',ignore_discard=True,ignore_expires=True)
handler = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(handler)
response = opener.open('http://www.baidu.com')
print(response.read().decode('utf-8'))
"""

# 处理异常
## URLError
"""
from urllib import request,error
try:
    response = request.urlopen('https://cuiqingcai.com/index.htm')
except error.URLError as e:
    print(e.reason)
"""
## HTTPError
"""
from urllib import request,error
try:
    response = request.urlopen('https://cuiqingcai.com/index.htm')
except error.HTTPError as e:
    print(e.reason,e.code,e.headers,sep='\n')
"""
"""
from urllib import request,error

try:
    response = request.urlopen('https://cuiqing.com/index.htm')
except error.HTTPError as e:
    print(e.reason,e.code,e.headers,sep='\n')
except error.URLError as e:
    print(e.reason)
else:
    print('Request Seccesfully')
"""
"""
import socket
import urllib.request
import urllib.error

try:
    response = urllib.request.urlopen('https://www.baidu.com',timeout=0.01)
except urllib.error.URLError as e:
    print(type(e.reason))
    if isinstance(e.reason,socket.timeout):
        print('TIME OUT')
"""

# 解析链接
"""
from urllib.parse import urlparse

result = urlparse('http://www.baidu.com/index.html;user?id=5#comment')
print(type(result),result)
"""
"""
from urllib.parse import urlparse
result = urlparse('www.baidu.com/index.html;user?id=5#comment',scheme='https')
print(result)
"""
"""
from urllib.parse import urlparse

result = urlparse('http://www.baidu.com/index.html;user?id=5#comment',allow_fragments=False)
print(type(result))
print(result)
result = urlparse('http://www.baidu.com/index.html;#comment',allow_fragments=False)
print(type(result))
print(result)
result = urlparse('https://www.baidu.com/index.html#comment',allow_fragments=False)
print('-----------------')
print(result.scheme,result[0],result.netloc,result[1],sep='\n')
"""
"""
from urllib.parse import urlunparse

data = ['http','www.baidu.com','index.html','user','a=6','comment']
print(urlunparse(data))
"""
"""
from urllib.parse import urlsplit

result = urlsplit('http://www.baidu.com/index.html;user?id=5#comment')
print(result)
print(result[0])
"""
"""
from urllib.parse import urlunsplit

data = ['http','www.baidu.com','index.html','a=6','comment']
print(urlunsplit(data))
"""
"""
from urllib.parse import urljoin

print(urljoin('http://www.baidu.com','FAQ.html'))
print(urljoin('http://www.baidu.com','https://cuiqingcai.com/FAQ.html'))
print(urljoin('http://www.baidu.com/about.html','https://cuiqingcai.com/FAQ.html'))
print(urljoin('http://www.baidu.com/about.html','https://cuiqingcai.com/FAQ.html?question=2'))
print(urljoin('http://www.baidu.com?wd=abc','https://cuiqingcai.com/index.php'))
print(urljoin('http://www.baidu.com','?category=2?#comment'))
print(urljoin('http://www.baidu.com','?category=2#comment'))
print(urljoin('http://www.baidu.com','?category=2'))
"""
"""
from urllib.parse import urlencode

params = {
    'name': 'germey',
    'age': 22
}

base_url = 'http:www.baidu.com?'
url = base_url + urlencode(params)
print(url)
"""
"""
from urllib.parse import parse_qs

query = 'name=germey&age=22'
print(parse_qs(query))
"""
"""
from urllib.parse import parse_qsl

query = 'name=germey&age=22'
print(parse_qsl(query))
"""
"""
from urllib.parse import quote

keyword = '壁纸'
url = 'https://www.baidu.com/s?wd=' + quote(keyword)
print(url)
"""
"""
from urllib.parse import unquote

url = 'https://www.baidu.com/s?wd=%E5%A3%81%E7%BA%B8'
print(unquote(url))
"""

# 分析Robots协议
"""
from urllib.robotparser import RobotFileParser

rp = RobotFileParser()
rp.set_url('http://www.jianshu.com/robots.txt')
rp.read()
print(rp.can_fetch('*','https://www.jianshu.com/p/b67554025d7d'))
print(rp.can_fetch('*',"http://www.jianshu.com/search?`=python&page=1&type=collections"))
"""
"""
from urllib.robotparser import RobotFileParser
from urllib.request import urlopen
from urllib.error import HTTPError

rp = RobotFileParser()
try:
    rp.parse(urlopen('https://www.jianshu.com/robots.txt').read().decode('utf-8').split('\n'))
    print(rp.can_fetch('*', 'https://www.jianshu/p/b67554025d7d'))
    print(rp.can_fetch('*', 'https://jianshu.com/search?q=python&page=1&type=collections'))
except HTTPError as e:
    print(e.reason)
"""






